/*package com.example.demo.util.craw.webmagic;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.processor.example.ZhihuPageProcessor;

*//**
 * Description: 第一个使用webmagic的爬虫程序<br/>
 *
 * @author liulonglong<br/>;
 * @taskId <br/>
 * @Date 2019/5/29 11:25
 * @Version 1.0
 *//*
public class GithubRepoPageProcessor implements PageProcessor{

        private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);

        @Override
        public void process(Page page) {
            page.addTargetRequests(page.getHtml().links().regex("https://www\\.zhihu\\.com/question/\\d+/answer/\\d+.*").all());
            page.putField("title", page.getHtml().xpath("//h1[@class='QuestionHeader-title']/text()").toString());
            page.putField("question", page.getHtml().xpath("//div[@class='QuestionRichText']//tidyText()").toString());
            page.putField("answer", page.getHtml().xpath("//div[@class='QuestionAnswer-content']/tidyText()").toString());
            if (page.getResultItems().get("title")==null){
                //skip this page
                page.setSkip(true);
            }
        }

        @Override
        public Site getSite() {
            return site;
        }

        public static void main(String[] args) {
            Spider.create(new us.codecraft.webmagic.processor.example.ZhihuPageProcessor()).addUrl("https://www.zhihu.com/explore").run();
        }

}
*/